{
 "cells": [
  {
   "cell_type": "code",
   "id": "843482d8-4944-4de7-9ea1-62a089cb0849",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-15T08:01:16.595151Z",
     "start_time": "2025-08-15T08:01:16.550603Z"
    }
   },
   "source": [
    "import numpy as np     #只需要下载numpy库即可\n",
    "import random\n",
    "import GridWorld_v2"
   ],
   "outputs": [],
   "execution_count": 1
  },
  {
   "cell_type": "code",
   "id": "864c3b7f-fea4-4caa-8ae5-dce3516b4b95",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-15T08:01:16.607155Z",
     "start_time": "2025-08-15T08:01:16.596154Z"
    }
   },
   "source": [
    "gamma = 0.95   #折扣因子，越接近0越近视\n",
    "\n",
    "rows = 5      #记得行数和列数这里要同步改\n",
    "columns = 5\n",
    "\n",
    "# gridworld = GridWorld_v2.GridWorld_v2(rows=rows, columns=columns, forbiddenAreaNums=8, targetNums=2, seed = 52,forbiddenAreaScore=-10)\n",
    "# gridworld = GridWorld_v2.GridWorld_v2(desc = [\".#\",\".T\"])             #赵老师4-1的例子\n",
    "# gridworld = GridWorld_v2.GridWorld_v2(desc = [\"##.T\",\"...#\",\"....\"])  #随便弄的例子\n",
    "gridworld = GridWorld_v2.GridWorld_v2(forbiddenAreaScore=-10, score=1,desc = [\".....\",\".##..\",\"..#..\",\".#T#.\",\".#...\"]) \n",
    "#gridworld = GridWorld_v2(forbiddenAreaScore=-10, score=1,desc = [\"T.\"]) \n",
    "gridworld.show()\n",
    "\n",
    "\n",
    "value = np.zeros(rows*columns)       #初始化可以任意，也可以全0\n",
    "qtable = np.zeros((rows*columns,5))  #初始化，这里主要是初始化维数，里面的内容会被覆盖所以无所谓\n",
    "\n",
    "\n",
    "# np.random.seed(50)\n",
    "policy = np.eye(5)[np.random.randint(0,5,size=(rows*columns))] \n",
    "gridworld.showPolicy(policy)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "⬜️⬜️⬜️⬜️⬜️\n",
      "⬜️🚫🚫⬜️⬜️\n",
      "⬜️⬜️🚫⬜️⬜️\n",
      "⬜️🚫✅🚫⬜️\n",
      "⬜️🚫⬜️⬜️⬜️\n",
      "⬆️🔄🔄⬅️⬆️\n",
      "⬆️⏩️🔄⬅️➡️\n",
      "➡️➡️⏫️⬅️➡️\n",
      "🔄⏬✅⏫️⬆️\n",
      "⬆️⏩️⬆️⬆️➡️\n"
     ]
    }
   ],
   "execution_count": 2
  },
  {
   "cell_type": "code",
   "id": "83befbda-7855-44ca-b31f-47cf5a074cf9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-15T08:01:16.610022Z",
     "start_time": "2025-08-15T08:01:16.607658Z"
    }
   },
   "source": [
    "from IPython.display import clear_output"
   ],
   "outputs": [],
   "execution_count": 3
  },
  {
   "cell_type": "code",
   "id": "3689097d-fac9-45af-aa1a-a883eff8837e",
   "metadata": {},
   "source": [
    "# 通过采样的方法计算action value，model free的话意味着不知道整个gridworld的概率了，所以不能直接套贝尔曼方程迭代求解\n",
    "#这个代码是非常难调的，很难调收敛\n",
    "policy = np.eye(5)[np.random.randint(0,5,size=(rows*columns))]  # 随机初始化策略，使用独热编码表示每个状态的动作概率分布\n",
    "gridworld.show()  # 显示网格世界\n",
    "gridworld.showPolicy(policy)  # 显示当前策略\n",
    "print(\"random policy\")  # 打印提示信息，表示当前是随机策略\n",
    "trajectorySteps = 5000  # 定义每个轨迹的步数\n",
    "epsilon = 0.1  # 初始化探索率，原注释中提到也可设为0.2，这个是超参数，可以自行调整\n",
    "qtable = np.zeros((rows*columns,5))  # 生成Q表，即动作价值表\n",
    "num_episodes = 400  # 定义训练的回合数\n",
    "epsilon_start = 0.3\n",
    "epsilon_end = 0.05\n",
    "epsilon_decay_rate = 0.995\n",
    "for episode in range(num_episodes):  # 开始迭代每个回合\n",
    "    #探索率指数衰减\n",
    "    # if(epsilon > 0.001) :  # 如果探索率大于0.001\n",
    "    #     epsilon -= 0.001  # 逐渐降低探索率\n",
    "    # else:\n",
    "    #     epsilon = 0.001  # 确保探索率不低于0.001\n",
    "    epsilon = max(epsilon_end, epsilon_start * (epsilon_decay_rate ** episode))    \n",
    "    p1 = 1-epsilon * (4/5)  # 计算选择最优动作的概率\n",
    "    p0 = epsilon/5  # 计算选择非最优动作的概率\n",
    "    # trajectorySteps = int(20+epsilon*1000)\n",
    "    print(\"trajectorySteps\",trajectorySteps)  # 打印当前轨迹步数\n",
    "    print(f\"epision:{epsilon}, p1:{p1}, p0:{p0}\")  # 打印当前探索率、选择最优动作和非最优动作的概率\n",
    "    \n",
    "    d = {1:p1, 0:p0}  # 创建一个字典，用于将策略中的0和1映射到对应的概率\n",
    "    policy_epsilon = np.vectorize(d.get)(policy)  # 将策略转换为epsilon-greedy策略，也就是原本是0和1，现在变为概率值\n",
    "    \n",
    "    i = random.randint(0,24)  # 随机选择初始状态\n",
    "    j = random.randint(0,4)  # 随机选择初始动作\n",
    "    cnt = [0 for i in range(25)]  # 初始化每个状态的访问次数计数器\n",
    "    qtable_rewards = [[0 for j in range(5)] for i in range(rows * columns)]  # 初始化每个状态-动作对的累计奖励，全是0\n",
    "    qtable_nums =    [[0 for j in range(5)] for i in range(rows * columns)]  # 初始化每个状态-动作对的访问次数，全是0\n",
    "    Trajectory = gridworld.getTrajectoryScore(nowState=i, action=j, policy=policy_epsilon, steps=trajectorySteps)  # 获取轨迹及其得分\n",
    "    clear_output(wait=True)  # 清除输出，等待新的输出\n",
    "    # 注意这里的返回值是大小为(trajectorySteps+1)的元组列表，因为把第一个动作也加入进去了\n",
    "    score = 0  # 初始化累计折扣奖励\n",
    "    #下面的代码和5.2是完全一样的\n",
    "    for k in range(trajectorySteps,-1,-1):  # 从轨迹的最后一步开始逆向遍历\n",
    "        tmpstate, tmpaction, tmpscore, _, __  = Trajectory[k]  # 解包轨迹中的状态、动作和得分\n",
    "        cnt[tmpstate] += 1  # 对应状态的访问次数加1\n",
    "        score = score*gamma + tmpscore  # 计算累计折扣奖励，从后往前更新\n",
    "        qtable_rewards[tmpstate][tmpaction] += score  # 对应状态-动作对的累计奖励加上当前累计折扣奖励\n",
    "        qtable_nums[tmpstate][tmpaction] += 1  # 对应状态-动作对的访问次数加1\n",
    "        qtable[tmpstate][tmpaction] = qtable_rewards[tmpstate][tmpaction] / qtable_nums[tmpstate][tmpaction]  # 更新Q表\n",
    "    values = []  # 初始化状态价值列表\n",
    "    for i in range(25):  # 遍历每个状态\n",
    "        v = 0  # 初始化状态价值\n",
    "        for j in range(5):  # 遍历每个动作\n",
    "            v += policy_epsilon[i][j] * qtable[i][j]  # 计算状态价值\n",
    "        values.append(v)  # 将状态价值添加到列表中\n",
    "    print(np.array(values).reshape(5,5))  # 打印状态价值矩阵，调试使用\n",
    "    \n",
    "    # print(qvalue.reshape(5,5))\n",
    "    gridworld.showPolicy(policy)\n",
    "    print(np.array(values).mean())\n",
    "    \n",
    "    \n",
    "    policy = np.eye(5)[np.argmax(qtable,axis=1)]  #qtable的最优值作为更新策略，并用独热码来表示\n",
    "    policy_epsilon = np.vectorize(d.get)(policy)  #将策略转换为epsilon-greedy策略，也就是原本是0和1，现在变为概率值\n",
    "        \n",
    "    # print(np.array(cnt).reshape(5,5))\n",
    "\n",
    "\n",
    "    \n",
    "\n",
    "\n",
    "    "
   ],
   "execution_count": 4,
   "outputs": []
  },
  {
   "cell_type": "code",
   "id": "6da1ce49-9b72-44a9-9413-c811bb04f577",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-15T08:01:58.897535Z",
     "start_time": "2025-08-15T08:01:58.894130Z"
    }
   },
   "source": [
    "\n",
    "gridworld.showPolicy(policy)\n",
    "\n",
    "\n",
    "  # 显示当前策略#打印状态价值的平均值 根据Q表编  # 将更新后的策略转换为epsilon-greedy策略"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "⬆️➡️🔄➡️🔄\n",
      "⬇️⏩️⏩️➡️🔄\n",
      "⬇️➡️⏩️➡️⬆️\n",
      "⬇️⏪✅⏬🔄\n",
      "⬆️🔄➡️➡️⬆️\n"
     ]
    }
   ],
   "execution_count": 5
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
